{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!sudo apt-get install -y cmake build-essential pkg-config libgoogle-perftools-dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -r sentencepiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!git clone https://github.com/google/sentencepiece.git "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd sentencepiece && mkdir build && cd build && cmake .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd sentencepiece &&  cd build && make -j $(nproc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd sentencepiece &&  cd build && make install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd sentencepiece &&  cd build && ldconfig -v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!spm_train --help"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-09-23T10:55:19.838669Z",
     "iopub.status.busy": "2023-09-23T10:55:19.838257Z",
     "iopub.status.idle": "2023-09-23T10:55:33.924760Z",
     "shell.execute_reply": "2023-09-23T10:55:33.923358Z",
     "shell.execute_reply.started": "2023-09-23T10:55:19.838636Z"
    }
   },
   "outputs": [],
   "source": [
    "!pip install sentencepiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!spm_train --input=./book_sdataset/booksdataset.txt --model_prefix=./my-tokenizer --vocab_size=32000 --character_coverage=0.9995 --model_type=bpe --shuffle_input_sentence=true\n",
    "#input放入你的训练数据目录\n",
    "#model_prefix放入你的输出路径目录\n",
    "#vocab_size词表大小"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head -n200 /kaggle/working/chinese-tokenizer.vocab"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}